import hashlib
import json
import random
from datetime import datetime
import time

import execjs
import pymongo
import redis
import requests
import pprint


class Trade:
    def __init__(self):
        self.mongo_client = pymongo.MongoClient(host='127.0.0.1', port=27017)
        self.collection = self.mongo_client['py_spider']['gd_trade_info']
        self.redis_client = redis.Redis(host='127.0.0.1', port=6379, db=1)
        self.headers = {
            "Accept": "application/json, text/plain, */*",
            "Accept-Language": "zh-CN,zh;q=0.9",
            "Cache-Control": "no-cache",
            "Connection": "keep-alive",
            "Content-Type": "application/json",
            "Origin": "https://ygp.gdzwfw.gov.cn",
            "Pragma": "no-cache",
            "Referer": "https://ygp.gdzwfw.gov.cn/",
            "Sec-Fetch-Dest": "empty",
            "Sec-Fetch-Mode": "cors",
            "Sec-Fetch-Site": "same-origin",
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36",
            "X-Dgi-Req-App": "ggzy-portal",
        }

        self.js = execjs.compile(open('公共资源交易.js', encoding='utf-8').read())

        self.url = "https://ygp.gdzwfw.gov.cn/ggzy-portal/search/v2/items"

    def get_data(self, page):
        data = {
            "type": "trading-type",
            "openConvert": False,
            "keyword": "",
            "siteCode": "44",
            "secondType": "A",
            "tradingProcess": "",
            "thirdType": "[]",
            "projectType": "",
            "publishStartTime": "",
            "publishEndTime": "",
            "pageNo": page,
            "pageSize": 10
        }

        t = str(int(time.time() * 1000))
        p = self.js.call('get_data', data, int(t))
        # print(p)
        self.headers['X-Dgi-Req-Nonce'] = p[0]
        self.headers['X-Dgi-Req-Signature'] = p[1]
        self.headers['X-Dgi-Req-Timestamp'] = t

        response = requests.post(self.url, headers=self.headers, json=data)

        result = response.json()
        print(f"获取第{page}页数据：", response.status_code)
        for res in result['data']['pageData']:
            item = dict()
            item['title'] = res['noticeTitle']
            item['date'] = datetime.strptime(res['publishDate'], "%Y%m%d%H%M%S").strftime("%Y-%m-%d %H:%M:%S")
            hash_value = hashlib.md5(json.dumps(item, ensure_ascii=False).encode()).hexdigest()
            if self.redis_client.sadd("gd_trade_filter", hash_value) != 0:
                self.collection.insert_one(item)
                print("入库成功：", item)
            else:
                print("数据重复：")


if __name__ == '__main__':
    t = Trade()
    for i in range(1, 1001):
        t.get_data(i)
        time.sleep(random.randint(1, 3))
